const cheerio = require('cheerio');
const superagent = require('superagent');
const fs = require('fs')
const path = require('path')
const charset = require("superagent-charset");
charset(superagent);

const decodeConfig = { decodeEntities: false }
const dataRoot = './data/CN'
const homePath = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020'
var provinceRequests = []
const lastDir = getLastDir('')

// 获取省级数据
function getProvince(url) {
  // 获取省级列表界面
  superagent.get(url).charset('gb2312').buffer(true).end(function (err, sres) {
    if (err) {
      console.log(url, err)
      return
    }
    const $ = cheerio.load(sres.text, decodeConfig);
    
    // 获取省级元素
    $(".provincetr td a").each(function (index, ele) {
      // 11.html
      const href = $(ele).attr('href')
      provinceRequests.push({
        url: `${homePath}/${href}`,
        dir: href.replace('.html', '')
      })
    })
    
    if (provinceRequests.length != 0) {
      handleProvince();
    }
  })
}

// 处理省级元素
async function handleProvince () {
  let provinceRequest = provinceRequests.shift()
  const lastProvince = lastDir.split('/')[1]

  // 如果省code小于上次同步的省的code，则跳过
  while (provinceRequest && provinceRequest.dir < lastProvince) {
    provinceRequest = provinceRequests.shift()
  }
  if (!provinceRequest) {
    return
  }
  if (!fs.existsSync(`${dataRoot}/${provinceRequest.dir}`)) {
    fs.mkdirSync(`${dataRoot}/${provinceRequest.dir}`)
  }
  // 获取省下面的地区数据
  await getRegion(provinceRequest.url, `/${provinceRequest.dir}`)
  console.log(`省${provinceRequest.dir}同步完成，5分钟后继续同步下个省`)
  setTimeout(handleProvince, 300000)
}

// 获取地区数据
async function getRegion(url, dir) {
  if (!fs.existsSync(dataRoot + dir)) {
    fs.mkdirSync(dataRoot + dir)
  }
  await sleep(500);
  // 获取地区列表界面
  const sres = await superagent.get(url).charset('gb2312').buffer(true);
  console.log('get Data dir')
  if (sres.statusCode != 200) {
    console.log(sres)
    throw '获取数据异常'
  }
  
  const $ = cheerio.load(sres.text, decodeConfig);
  let items = [];

  // 获取地区元素
  const trList = $(".citytr,.countytr,.towntr,.villagetr");
  for (let index = 0; index < trList.length; index ++) {
    const tdList = $(trList[index]).children('td');
    const code = $(tdList[0]).text()
    const cnName = $(tdList[tdList.length - 1]).text()
    items.push({
      code,
      cnName
    });
    // 如果地区下面还有子地区,则获取子地区
    if (tdList.children('a').length > 0) {
      // 13/1301.html,01/130102.html,02/130102001.html，获取后面的html文件名
      const regionPage = $(tdList[0]).children('a').attr('href').split('/').pop();
      // 将前面的父编码替换成空，提取01，02，001编码
      const simpleCode = regionPage.replace(dir.replace(/\//g, ''), '').replace('.html', '');
      const dirNew = `${dir}/${simpleCode}` ;
      // 如果地区目录小于上次同步的地区目录则跳过获取子地区
      if (dirNew > lastDir || lastDir.startsWith(dirNew)) {
        await getRegion(
          `${homePath}${dir}/${regionPage}`,
          dirNew
        )
      }
    }
  }
  const file = path.resolve(__dirname, dataRoot + dir + '/data.json')
  fs.writeFile(file, JSON.stringify(items), { encoding: 'utf8' }, err => {
    if (err != null) {
      console.log('writefFile', err)
    }
  })
}

// 获取上一次正在同步的目录
function getLastDir (parentDir) {
  const subs = fs.readdirSync(dataRoot + parentDir, { withFileTypes: true });
  
  const lastDir = subs.filter(sub=>sub.isDirectory()).pop();
  if (lastDir) {
    return getLastDir(parentDir + '/' + lastDir.name)
  } else {
    return parentDir
  }
}

getProvince(homePath + '/index.html')
console.log('上一次同步的地区', lastDir)

function sleep(time = 0) {
  return new Promise((resolve, reject) => {
    setTimeout(() => {
      resolve();
    }, time);
  })
}